<!DOCTYPE html>
<html lang="zh-cn">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 5.4.0">
  <link rel="icon" type="image/png" sizes="16x16" href="https://gitee.com/reku1997/reku1997/raw/master/reku.ico">

<link rel="stylesheet" href="/css/main.css">


<link rel="stylesheet" href="/lib/font-awesome/css/font-awesome.min.css">

<script id="hexo-configurations">
    var NexT = window.NexT || {};
    var CONFIG = {"hostname":"reku1997.gitee.io","root":"/","scheme":"Gemini","version":"7.8.0","exturl":false,"sidebar":{"position":"left","display":"post","padding":18,"offset":12,"onmobile":false},"copycode":{"enable":true,"show_result":true,"style":"flat"},"back2top":{"enable":true,"sidebar":false,"scrollpercent":false},"bookmark":{"enable":false,"color":"#222","save":"auto"},"fancybox":false,"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"algolia":{"appID":"AW5K8S9IEE","apiKey":"d7e666d597854738d2fb31ecaa989aa5","indexName":"dev_reku1997","hits":{"per_page":10},"labels":{"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}},"localsearch":{"enable":false,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},"motion":{"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}}};
  </script>

  <meta name="description" content="课程网站 在头条 AML 实习的时候就觉得这个 AI system 方向非常有趣，但是苦于实验室不是搞这一套的，自己拖延症也非常严重，所以一直在入门的边缘徘徊。但是在今天——研一秋学期考试周的前一周，我决定开始学习 AI system 方向最著名的必学课程，Tianqi Chen 在 UW 开设的 CSE599W。 这个课程其实资料并不是很完善，只有 github 上面的几个 repo 和课程网站">
<meta property="og:type" content="article">
<meta property="og:title" content="CSE 599W: Systems for ML 课程笔记 1-6">
<meta property="og:url" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/index.html">
<meta property="og:site_name" content="Reku">
<meta property="og:description" content="课程网站 在头条 AML 实习的时候就觉得这个 AI system 方向非常有趣，但是苦于实验室不是搞这一套的，自己拖延症也非常严重，所以一直在入门的边缘徘徊。但是在今天——研一秋学期考试周的前一周，我决定开始学习 AI system 方向最著名的必学课程，Tianqi Chen 在 UW 开设的 CSE599W。 这个课程其实资料并不是很完善，只有 github 上面的几个 repo 和课程网站">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/3-1.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/3-2.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/3-3.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/3-4.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/3-5.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/3-6.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/3-7.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/3-8.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/3-9.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/3-10.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/4-1.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/4-2.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/4-3.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/4-5.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/4-4.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/5-1.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/5-2.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/5-3.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/5-3.5.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/5-3.8.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/5-4.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/5-5.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/5-6.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/5-7.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/6-1.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/6-2.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/6-3.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/6-4.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/6-5.png">
<meta property="og:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/6-6.png">
<meta property="article:published_time" content="2019-11-08T07:31:51.000Z">
<meta property="article:modified_time" content="2021-12-16T11:33:12.000Z">
<meta property="article:author" content="Reku">
<meta property="article:tag" content="learning">
<meta property="article:tag" content="system">
<meta property="article:tag" content="课程笔记">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/3-1.png">

<link rel="canonical" href="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome : false,
    isPost : true,
    lang   : 'zh-cn'
  };
</script>

  <title>CSE 599W: Systems for ML 课程笔记 1-6 | Reku</title>
  






  <noscript>
  <style>
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

<link rel="alternate" href="/atom.xml" title="Reku" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container use-motion">
    <div class="headband"></div>

    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-nav-toggle">
    <div class="toggle" aria-label="Toggle navigation bar">
      <span class="toggle-line toggle-line-first"></span>
      <span class="toggle-line toggle-line-middle"></span>
      <span class="toggle-line toggle-line-last"></span>
    </div>
  </div>

  <div class="site-meta">

    <a href="/" class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <h1 class="site-title">Reku</h1>
      <span class="logo-line-after"><i></i></span>
    </a>
  </div>

  <div class="site-nav-right">
    <div class="toggle popup-trigger">
        <i class="fa fa-search fa-fw fa-lg"></i>
    </div>
  </div>
</div>




<nav class="site-nav">
  <ul id="menu" class="menu">
        <li class="menu-item menu-item-home">

    <a href="/" rel="section"><i class="fa fa-fw fa-home"></i>Home</a>

  </li>
        <li class="menu-item menu-item-about">

    <a href="/about/" rel="section"><i class="fa fa-fw fa-user"></i>About</a>

  </li>
        <li class="menu-item menu-item-tags">

    <a href="/tags/" rel="section"><i class="fa fa-fw fa-tags"></i>Tags</a>

  </li>
        <li class="menu-item menu-item-archives">

    <a href="/archives/" rel="section"><i class="fa fa-fw fa-archive"></i>Archives</a>

  </li>
        <li class="menu-item menu-item-sitemap">

    <a href="/sitemap.xml" rel="section"><i class="fa fa-fw fa-sitemap"></i>Sitemap</a>

  </li>
      <li class="menu-item menu-item-search">
        <a role="button" class="popup-trigger"><i class="fa fa-search fa-fw"></i>Search
        </a>
      </li>
  </ul>
</nav>



  <div class="search-pop-overlay">
    <div class="popup search-popup">
        <div class="search-header">
  <span class="search-icon">
    <i class="fa fa-search"></i>
  </span>
  <div class="search-input-container"></div>
  <span class="popup-btn-close">
    <i class="fa fa-times-circle"></i>
  </span>
</div>
<div class="algolia-results">
  <div id="algolia-stats"></div>
  <div id="algolia-hits"></div>
  <div id="algolia-pagination" class="algolia-pagination"></div>
</div>

      
    </div>
  </div>

</div>
    </header>

    
  <div class="back-to-top">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>


    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          

          <div class="content post posts-expand">
            

    
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="zh-cn">
    <link itemprop="mainEntityOfPage" href="https://reku1997.gitee.io/2019/11/08/cse-599w-systems-for-ml-1-6/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="Reku">
      <meta itemprop="description" content="">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Reku">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          CSE 599W: Systems for ML 课程笔记 1-6
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">Posted on</span>

              <time title="Created: 2019-11-08 15:31:51" itemprop="dateCreated datePublished" datetime="2019-11-08T15:31:51+08:00">2019-11-08</time>
            </span>
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                <span class="post-meta-item-text">Edited on</span>
                <time title="Modified: 2021-12-16 19:33:12" itemprop="dateModified" datetime="2021-12-16T19:33:12+08:00">2021-12-16</time>
              </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">In</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/system/" itemprop="url" rel="index"><span itemprop="name">system</span></a>
                </span>
                  , 
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/system/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0/" itemprop="url" rel="index"><span itemprop="name">机器学习</span></a>
                </span>
            </span>

          
            <span id="/2019/11/08/cse-599w-systems-for-ml-1-6/" class="post-meta-item leancloud_visitors" data-flag-title="CSE 599W: Systems for ML 课程笔记 1-6" title="Views">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">Views: </span>
              <span class="leancloud-visitors-count"></span>
            </span>
  
  <span class="post-meta-item">
    
      <span class="post-meta-item-icon">
        <i class="fa fa-comment-o"></i>
      </span>
      <span class="post-meta-item-text">Valine: </span>
    
    <a title="valine" href="/2019/11/08/cse-599w-systems-for-ml-1-6/#valine-comments" itemprop="discussionUrl">
      <span class="post-comments-count valine-comment-count" data-xid="/2019/11/08/cse-599w-systems-for-ml-1-6/" itemprop="commentCount"></span>
    </a>
  </span>
  
  

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
        <p><a target="_blank" rel="noopener" href="http://dlsys.cs.washington.edu/">课程网站</a> 在头条 AML 实习的时候就觉得这个 AI system 方向非常有趣，但是苦于实验室不是搞这一套的，自己拖延症也非常严重，所以一直在入门的边缘徘徊。但是在今天——研一秋学期考试周的前一周，我决定开始学习 AI system 方向最著名的必学课程，Tianqi Chen 在 UW 开设的 CSE599W。 这个课程其实资料并不是很完善，只有 github 上面的几个 repo 和课程网站上面的 slide，缺乏讲课的视频资源。而且在开始学习之前就听说很多地方 slide 写的非常简陋，只能通过 tvm 和 tinyflow 代码慢慢学习。我在学习之前也找了一些 blog 资源，开个坑，希望可以努力坚持下来！ 本人的作业也开源到 <a target="_blank" rel="noopener" href="https://github.com/wyc-ruiker/CSE-599W-2018">github</a> 上面了，希望大家多多指导。</p>
<span id="more"></span>
<p><a target="_blank" rel="noopener" href="http://jcf94.com/2018/10/04/2018-10-04-cse559w/">Chenfan Blog——CSE 599W： Systems for ML</a></p>
<p><a target="_blank" rel="noopener" href="https://zhuanlan.zhihu.com/c_186688444">知乎专栏：SysML/DL机器学习系统</a></p>
<p><a target="_blank" rel="noopener" href="https://zhuanlan.zhihu.com/p/50529704">手把手带你遨游TVM</a></p>
<p>不过上面这两个参考资料，第一个有点过于简略，第二个虽然写的很好但是已经五个月没有更新了...可能是坑了，所以后面的内容还是要靠自己慢慢研究了... 第三篇是蓝色大大的 TVM 入门文章，写的非常赞，可以清晰的理解 TVM 存在的意义与解决的问题。</p>
<h2 id="lecture-1-2">Lecture 1-2</h2>
<p>介绍了一波深度学习的基本概念，常见的各种 CNN、RNN、激活函数、BatchNormal、梯度消失与梯度爆炸都用一节课介绍了一通。因为我之前学过 Ng 在 coursera 上面的 Deep learing 专项，所以对这些都很了解了，而且这些内容的资源满大街都是，就不在这里继续介绍了。 第二讲是一个实验课，用 mxnet 搭建一个基础的网络，虽然我之前系统看过沐神的《动手深度学习》，但是我看的是 github 上面的 pytorch 魔改版，mxnet 只在上半年的华为软挑的时候用过一下，不是特别了解。但是这个 gluon api 似乎跟 pytorch 大同小异，这里也不多废话了，有兴趣自己看看原版《动手深度学习》就好了。</p>
<h2 id="lecture-3-overview-of-deep-learning-system">Lecture 3: Overview of Deep Learning System</h2>
<p>学习过什么是 Deep Learning 了，那么啥是 Deep Learning System 呢？ 在我的理解其实就是从调包到真正计算出结果的全过程，就是 Deep Learning System，也就是通常所说的算法的真正落地。 在这个课程中，Deep Learning System 从高层到底层分成了三个部分：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/3-1.png"></p>
<p>第一部分就是调包的封装 API，第二部分表示调完包后对调包代码的优化与 Scheduling，第三部分就是最下面的一些高效的 GPU kernel 实现、不同硬件后端的部署等等。 那么一个 Deep Learning 框架在 API 层需要包括什么内容呢？为什么大家要用框架而不是自己从头写呢？ 下面这个图就回答了这个问题，现在的模型越来越大，实现起来需要注意的内容也就越来越多，如果每步都由我们自己来进行链式求导算梯度的话，可能就没有这么多转行深度学习的大哥了（</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/3-2.png"></p>
<p>计算图是一个 Deep Learning 框架的基本概念，节点表示运算操作，边表示数据依赖，这里展示了一个最简单的 Logistic Regression 计算图实例： 首先是计算 loss 之前的前向传播，其实就是一个最简单的矩阵乘法加一个 softmax：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/3-3.png"></p>
<p>然后是将 softmax 输出的东西搞一个交叉熵作为 loss，相当于最大化 liklihood:</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/3-4.png"></p>
<p>然后是自动微分：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/3-5.png"></p>
<p>最后通过 SGD 来更新梯度：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/3-6.png"></p>
<p>结合上面的所有步骤，我们就得到了一个计算图：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/3-7.png"></p>
<p>最上层的 API 做了最简单的介绍，下面就是中间的 System 部分，讲的是对计算图的优化和对计算的调度。 计算优化最简单的一种就是 memory 优化，比如增加 cache 利用率之类的。因为我们的代码通常跑在多个线程甚至多个计算设备上面，所以并行调度也是非常重要的。最简单的一种并行调度如下图，这是一个 mxnet 代码，因为计算 C 和计算 B 是完全独立的，所以这两个部分可以并行化。</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/3-8.png"></p>
<p>然后简单介绍一下最底层的情况，我们训练完了 model，要部署到不同的设备上。想必大家也都听过各种各样的厂商搞出过各种各样的部署方法，为了产生更好的模型性能：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/3-9.png"></p>
<p>TVM 就是为了解决这样的问题而诞生的，只要我们都搞成中间代码，全栈自动编译优化，这样 model 就可以非常简单的部署到不同的设备上了：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/3-10.png"></p>
<p>这门课将在接下来详细介绍 Deep Learning System 的三个部分。</p>
<h2 id="lecture-4-backpropagation-and-automatic-differentiation">Lecture 4: Backpropagation and Automatic Differentiation</h2>
<p>第四节课讲的是 Deep Learning 中的求导方式——Auto-Diff. 首先我们要了解现代计算机实现求导通常有哪些方式。 第一种叫做 Symbolic Differentiation，如图所示：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/4-1.png"></p>
<p>通过程序来维护整个求导的式子，然后把变量带入就得到最终的导数。这种做法的缺点在于表达式是一个很难维护的东西，最后要维护的东西就会越来越多。正常 Deep Learning 框架显然不应该选择这种求导方式... 第二种叫做 Numerical Differentiation，这种求导形式非常简单，看起来很适合计算机：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/4-2.png"></p>
<p>但是有个很关键的问题是，这种求导方法要进行两次正向传播，跑起来很慢，而且误差会比较大。但是因为其实现方式的简单，所以这是一个非常好的 grad check 工具。Ng 在 DL 专项里面也是用这种方式去进行 grad check 的。 第三种叫做 Backpropagation，现代 Deep Learning 的核心：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/4-3.png"></p>
<p>虽然这种做法很适合计算机，效率也不错。但有一个关键的问题，我们在做正向传播的时候要记录中间结果，这样才能在后面进行反向传播，内存耗费很大。而且难以形成计算图，无法进行通用的并行化。 这时候就需要掏出第四种方法，叫做 Automatic Differentiation：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/4-5.png"></p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/4-4.png"></p>
<p>其实思路非常简单，看伪代码就看的出来。这种方法就是通过反拓扑序去生成一个反向的计算图，因为是计算图所以不用保存任何中间变量；也因为是计算图，所以可以进行通用的并行化与代码优化，两全其美！ 接下来就是 Auto-diff 的具体实现，也就是 assignment 1。这次作业的难度不是太大，前面就是一些算子的简单实现，后面有两段重要的代码：</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">for</span> node <span class="keyword">in</span> topo_order:</span><br><span class="line">    <span class="keyword">if</span> <span class="built_in">isinstance</span>(node.op, PlaceholderOp):</span><br><span class="line">        <span class="keyword">continue</span></span><br><span class="line">    input_vals = [node_to_val_map[x] <span class="keyword">for</span> x <span class="keyword">in</span> node.inputs]</span><br><span class="line">    res = node.op.compute(node, input_vals)</span><br><span class="line">    <span class="keyword">if</span> <span class="built_in">isinstance</span>(res, np.ndarray) == <span class="literal">False</span>:</span><br><span class="line">        res = np.array(res)</span><br><span class="line">    node_to_val_map[node] = res</span><br></pre></td></tr></table></figure>
<p>这段是前向传播的部分，记录每个节点计算出来的结果。前向传播需要正向拓扑序。</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">for</span> node <span class="keyword">in</span> reverse_topo_order:</span><br><span class="line">    grad = sum_node_list(node_to_output_grads_list[node])</span><br><span class="line">    node_to_output_grad[node] = grad</span><br><span class="line">    input_grads = node.op.gradient(node, grad)</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> <span class="built_in">range</span>(<span class="built_in">len</span>(node.inputs)):</span><br><span class="line">        node_to_output_grads_list[node.inputs[i]] = node_to_output_grads_list.get(node.inputs[i], [])</span><br><span class="line">        node_to_output_grads_list[node.inputs[i]].append(input_grads[i])</span><br></pre></td></tr></table></figure>
<p>这段是反向形成 Auto-diff 计算图的过程，跟伪代码很像。进一步解释一下：按照反拓扑序来遍历节点，计算到这个节点就代表着所有相关的梯度都计算完了。现在需要把相关的梯度都加起来，然后加起来的梯度作为这个节点向后面传播的梯度。整个过程很容易理解。</p>
<h2 id="lecture-5-gpu-programming">Lecture 5: GPU Programming</h2>
<p>这一讲涉及到的部分比较底层，虽然浙大本科的硬件三连质量非常高，但是对于 GPU 架构接触还是非常少的，又没有视频，只能尽力看一看了... 首先讲的是 CPU 的架构，很容易理解，CPU 跟内存有关的操作都很慢，所以才有各种指令预测、cache 优化算法之类的折磨着一代有一代的计算机学子：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/5-1.png"></p>
<p>而 GPU 则是给 CPU 加了一大堆计算资源，比普通的指令向量化更加强劲：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/5-2.png"></p>
<p>从这个图可以看出来，GPU 的寄存器数量非常大，所以他可以开很多线程去并行计算，但是从这个图也可以看出来，GPU 的 cache 小的可怜，所以 GPU 适用于那种轻内存读写、重计算的任务，也就是大量的并行计算任务。</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/5-3.png"></p>
<p>然后讲了 CUDA Programming Model，因为我完全不懂 CUDA 编程，所以后面的东西都是我瞎理解的，不一定对。 这个叫做 SIMT 的 Model 就是把几个 thread group 成一个 block，再把几个 block group 成一个 grid，block 可以以任何顺序在 GPU 上面调度。</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/5-3.5.png"></p>
<p>然后是个最简单的 cuda 程序——vector add。过程非常简单，跟操作系统里面查页表差不多，这里通过 block 下标和 thread 下标可以得到向量加法的下标，然后每个线程都执行一样的代码就可以了。</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/5-3.8.png"></p>
<p>后面讲了一个窗口 sum 的例子，因为他图画的不太好，所以有点难以理解，我尽力理解了一下：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/5-4.png"></p>
<p>最简单的实现方式是这样的：</p>
<figure class="highlight cpp"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta">#<span class="meta-keyword">define</span> RADIUS 3</span></span><br><span class="line"><span class="function">__global__ <span class="keyword">void</span> <span class="title">windowSumNaiveKernel</span><span class="params">(<span class="keyword">const</span> <span class="keyword">float</span>* A, <span class="keyword">float</span>* B, <span class="keyword">int</span> n)</span> </span></span><br><span class="line"><span class="function"></span>&#123;    </span><br><span class="line">    <span class="keyword">int</span> out_index = blockDim.x * blockIdx.x + threadIdx.x;    </span><br><span class="line">    <span class="keyword">int</span> in_index = out_index + RADIUS;    </span><br><span class="line">    <span class="keyword">if</span> (out_index &lt; n) &#123;</span><br><span class="line">        <span class="keyword">float</span> sum = <span class="number">0.</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="keyword">int</span> i = -RADIUS; i &lt;= RADIUS; ++i) &#123;</span><br><span class="line">            sum += A[in_index + i];        </span><br><span class="line">        &#125;</span><br><span class="line">        B[out_index] = sum;</span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p>为什么说他图画的不太好，因为他的图跟代码的下标其实是对应不上的。B[0] 的结果应该是从 A[0] 到 A[6] 的和才对，这样下标就和代码完全对应上了。 这个实现非常简单，所以有很大的优化空间。其中最脑残的地方就是每个线程都读了 7 次 A，这样对 GPU 这种小 cache 来说是非常残忍的。</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/5-5.png"></p>
<p>所以一个非常简单的优化就是从一个线程读 7 次 A，变成一个 block (假设有四个线程) 读 9 次 A。</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/5-6.png"></p>
<p>那么下面的实现就很好理解了：</p>
<figure class="highlight cpp"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line"><span class="function">__global__ <span class="keyword">void</span> <span class="title">windowSumKernel</span><span class="params">(<span class="keyword">const</span> <span class="keyword">float</span>* A, <span class="keyword">float</span>* B, <span class="keyword">int</span> n)</span> </span>&#123;</span><br><span class="line">    __shared__ <span class="keyword">float</span> temp[THREADS_PER_BLOCK + <span class="number">2</span> * RADIUS];</span><br><span class="line">    <span class="keyword">int</span> out_index = blockDim.x * blockIdx.x + threadIdx.x;</span><br><span class="line">    <span class="keyword">int</span> in_index = out_index + RADIUS;</span><br><span class="line">    <span class="keyword">int</span> local_index = threadIdx.x + RADIUS;</span><br><span class="line">    <span class="keyword">if</span> (out_index &lt; n) &#123;</span><br><span class="line">        temp[local_index] = A[in_index];</span><br><span class="line">        <span class="keyword">if</span> (threadIdx.x &lt; RADIUS) &#123;</span><br><span class="line">            temp[local_index - RADIUS] = A[in_index - RADIUS];</span><br><span class="line">            temp[local_index + THREADS_PER_BLOCK] = A[in_index+THREADS_PER_BLOCK];</span><br><span class="line">        &#125;</span><br><span class="line">        __syncthreads();</span><br><span class="line">        <span class="keyword">float</span> sum = <span class="number">0.</span>;</span><br><span class="line">        <span class="keyword">for</span> (<span class="keyword">int</span> i = -RADIUS; i &lt;= RADIUS; ++i) &#123;</span><br><span class="line">            sum += temp[local_index + i];</span><br><span class="line">        &#125;        </span><br><span class="line">        B[out_index] = sum;    </span><br><span class="line">    &#125;</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<p>其中比较奥妙的地方是 if (threadIdx.x &lt; RADIUS) 这个语法块，其实也不难理解，就是把两边的 RADIUS 部分充满。最后是一个简单的矩阵乘法程序：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/5-7.png"></p>
<h2 id="lecture-6-optimize-for-hardware-backends">Lecture 6: Optimize for Hardware Backends</h2>
<p>这一讲处于计算图和硬件之间。 首先讲了一些体系结构的基本知识，比如多级 cache 之类的，然后讲了矩阵乘法的 cache 优化，非常基础。 最简单的矩阵乘法如下图所示：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/6-1.png"></p>
<p>像上面这样直接做矩阵乘法，其实对 cache 的利用率并不高，更好的方式是把矩阵分成很多个小块：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/6-2.png"></p>
<p>分块的方法可以缩小从下级 cache 到寄存器的 cost，有人想那我直接把 v1、v2 高的大一点，这样不就可以优化更多了吗？ 但是通常来说，寄存器非常小，很难存下一整列或者一整行，所幸 CPU 的 cache 通常比较大，也许可以存下：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/6-3.png"></p>
<p>把对寄存器的优化和 L1 cache 的优化相结合，就得到了下面这种比较复杂的优化方式：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/6-4.png"></p>
<p>对于 CPU 来说，代码优化可能集中于 Reuse memory 上面，而 GPU 的优化则是集中于 Reuse among threads 上。（这段其实不太懂，我理解就是跟前一讲的那个 window sum 一样，优化复用不同线程的内存）</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/6-5.png"></p>
<p>对于计算图的代码优化，其实套路千变万化：</p>
<p><img src="/2019/11/08/cse-599w-systems-for-ml-1-6/6-6.png"></p>
<p>为了解决这样的优化问题，我们就需要本门课的核心内容——TVM 了！</p>

    </div>

    
    
    

      <footer class="post-footer">
          <div class="post-tags">
              <a href="/tags/learning/" rel="tag"># learning</a>
              <a href="/tags/system/" rel="tag"># system</a>
              <a href="/tags/%E8%AF%BE%E7%A8%8B%E7%AC%94%E8%AE%B0/" rel="tag"># 课程笔记</a>
          </div>

        


        
    <div class="post-nav">
      <div class="post-nav-item">
    <a href="/2019/07/26/adaboost/" rel="prev" title="从 AdaBoost 到 GBDT">
      <i class="fa fa-chevron-left"></i> 从 AdaBoost 到 GBDT
    </a></div>
      <div class="post-nav-item">
    <a href="/2019/11/21/cse-599w-systems-for-ml-7-12/" rel="next" title="CSE 599W: SYSTEMS FOR ML 课程笔记 7-12">
      CSE 599W: SYSTEMS FOR ML 课程笔记 7-12 <i class="fa fa-chevron-right"></i>
    </a></div>
    </div>
      </footer>
    
  </article>
  
  
  



          </div>
          
    <div class="comments" id="valine-comments"></div>

<script>
  window.addEventListener('tabs:register', () => {
    let { activeClass } = CONFIG.comments;
    if (CONFIG.comments.storage) {
      activeClass = localStorage.getItem('comments_active') || activeClass;
    }
    if (activeClass) {
      let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
      if (activeTab) {
        activeTab.click();
      }
    }
  });
  if (CONFIG.comments.storage) {
    window.addEventListener('tabs:click', event => {
      if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
      let commentClass = event.target.classList[1];
      localStorage.setItem('comments_active', commentClass);
    });
  }
</script>

        </div>
          
  
  <div class="toggle sidebar-toggle">
    <span class="toggle-line toggle-line-first"></span>
    <span class="toggle-line toggle-line-middle"></span>
    <span class="toggle-line toggle-line-last"></span>
  </div>

  <aside class="sidebar">
    <div class="sidebar-inner">

      <ul class="sidebar-nav motion-element">
        <li class="sidebar-nav-toc">
          Table of Contents
        </li>
        <li class="sidebar-nav-overview">
          Overview
        </li>
      </ul>

      <!--noindex-->
      <div class="post-toc-wrap sidebar-panel">
          <div class="post-toc motion-element"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#lecture-1-2"><span class="nav-number">1.</span> <span class="nav-text">Lecture 1-2</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#lecture-3-overview-of-deep-learning-system"><span class="nav-number">2.</span> <span class="nav-text">Lecture 3: Overview of Deep Learning System</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#lecture-4-backpropagation-and-automatic-differentiation"><span class="nav-number">3.</span> <span class="nav-text">Lecture 4: Backpropagation and Automatic Differentiation</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#lecture-5-gpu-programming"><span class="nav-number">4.</span> <span class="nav-text">Lecture 5: GPU Programming</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#lecture-6-optimize-for-hardware-backends"><span class="nav-number">5.</span> <span class="nav-text">Lecture 6: Optimize for Hardware Backends</span></a></li></ol></div>
      </div>
      <!--/noindex-->

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
  <p class="site-author-name" itemprop="name">Reku</p>
  <div class="site-description" itemprop="description"></div>
</div>
<div class="site-state-wrap motion-element">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/archives/">
        
          <span class="site-state-item-count">78</span>
          <span class="site-state-item-name">posts</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/categories/">
        <span class="site-state-item-count">8</span>
        <span class="site-state-item-name">categories</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/tags/">
          
        <span class="site-state-item-count">96</span>
        <span class="site-state-item-name">tags</span></a>
      </div>
  </nav>
</div>
  <div class="links-of-author motion-element">
      <span class="links-of-author-item">
        <a href="https://github.com/wyc-ruiker" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;wyc-ruiker" rel="noopener" target="_blank"><i class="fa fa-fw fa-github"></i>GitHub</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://www.zhihu.com/people/reku1997" title="ZhiHu → https:&#x2F;&#x2F;www.zhihu.com&#x2F;people&#x2F;reku1997" rel="noopener" target="_blank"><i class="fa fa-fw fa-quora"></i>ZhiHu</a>
      </span>
      <span class="links-of-author-item">
        <a href="http://codeforces.com/profile/reku" title="CodeForces → http:&#x2F;&#x2F;codeforces.com&#x2F;profile&#x2F;reku" rel="noopener" target="_blank"><i class="fa fa-fw fa-code"></i>CodeForces</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://www.linkedin.cn/injobs/in/reku" title="Linkedin → https:&#x2F;&#x2F;www.linkedin.cn&#x2F;injobs&#x2F;in&#x2F;reku" rel="noopener" target="_blank"><i class="fa fa-fw fa-linkedin"></i>Linkedin</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://gitee.com/reku1997" title="Gitee → https:&#x2F;&#x2F;gitee.com&#x2F;reku1997" rel="noopener" target="_blank"><i class="fa fa-fw fa-github"></i>Gitee</a>
      </span>
      <span class="links-of-author-item">
        <a href="/./atom.xml" title="RSS → .&#x2F;atom.xml"><i class="fa fa-fw fa-rss"></i>RSS</a>
      </span>
  </div>



      </div>

    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


      </div>
    </main>

    <footer class="footer">
      <div class="footer-inner">
        

        

<div class="copyright">
  
  &copy; 2016 – 
  <span itemprop="copyrightYear">2022</span>
  <span class="with-love">
    <i class="fa fa-user"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">Reku</span>
</div>
  <div class="powered-by">Powered by <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Gemini</a>
  </div>

        
<div class="busuanzi-count">
  <script async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
    <span class="post-meta-item" id="busuanzi_container_site_uv" style="display: none;">
      <span class="post-meta-item-icon">
        <i class="fa fa-user"></i>
      </span>
      <span class="site-uv" title="Total Visitors">
        <span id="busuanzi_value_site_uv"></span>
      </span>
    </span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item" id="busuanzi_container_site_pv" style="display: none;">
      <span class="post-meta-item-icon">
        <i class="fa fa-eye"></i>
      </span>
      <span class="site-pv" title="Total Views">
        <span id="busuanzi_value_site_pv"></span>
      </span>
    </span>
</div>








      </div>
    </footer>
  </div>

  
  <script src="/lib/anime.min.js"></script>
  <script src="/lib/velocity/velocity.min.js"></script>
  <script src="/lib/velocity/velocity.ui.min.js"></script>

<script src="/js/utils.js"></script>

<script src="/js/motion.js"></script>


<script src="/js/schemes/pisces.js"></script>


<script src="/js/next-boot.js"></script>




  




  
<script src="//cdn.jsdelivr.net/npm/algoliasearch@4/dist/algoliasearch-lite.umd.js"></script>
<script src="//cdn.jsdelivr.net/npm/instantsearch.js@4/dist/instantsearch.production.min.js"></script>
<script src="/js/algolia-search.js"></script>














  

  
      

<script>
  if (typeof MathJax === 'undefined') {
    window.MathJax = {
      loader: {
        source: {
          '[tex]/amsCd': '[tex]/amscd',
          '[tex]/AMScd': '[tex]/amscd'
        }
      },
      tex: {
        inlineMath: {'[+]': [['$', '$']]},
        tags: 'ams'
      },
      options: {
        renderActions: {
          findScript: [10, doc => {
            document.querySelectorAll('script[type^="math/tex"]').forEach(node => {
              const display = !!node.type.match(/; *mode=display/);
              const math = new doc.options.MathItem(node.textContent, doc.inputJax[0], display);
              const text = document.createTextNode('');
              node.parentNode.replaceChild(text, node);
              math.start = {node: text, delim: '', n: 0};
              math.end = {node: text, delim: '', n: 0};
              doc.math.push(math);
            });
          }, '', false],
          insertedScript: [200, () => {
            document.querySelectorAll('mjx-container').forEach(node => {
              let target = node.parentNode;
              if (target.nodeName.toLowerCase() === 'li') {
                target.parentNode.classList.add('has-jax');
              }
            });
          }, '', false]
        }
      }
    };
    (function () {
      var script = document.createElement('script');
      script.src = '//cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js';
      script.defer = true;
      document.head.appendChild(script);
    })();
  } else {
    MathJax.startup.document.state(0);
    MathJax.texReset();
    MathJax.typeset();
  }
</script>

    

  


<script>
NexT.utils.loadComments(document.querySelector('#valine-comments'), () => {
  NexT.utils.getScript('//unpkg.com/valine/dist/Valine.min.js', () => {
    var GUEST = ['nick', 'mail', 'link'];
    var guest = 'nick,mail,link';
    guest = guest.split(',').filter(item => {
      return GUEST.includes(item);
    });
    new Valine({
      el         : '#valine-comments',
      verify     : false,
      notify     : false,
      appId      : 'MWLzM550UOu69h3dgvbbLSsF-gzGzoHsz',
      appKey     : 'gkKnwm9FK0cu3ysJbcggsCDz',
      placeholder: "Just go go",
      avatar     : 'mm',
      meta       : guest,
      pageSize   : '10' || 10,
      visitor    : true,
      lang       : '' || 'zh-cn',
      path       : location.pathname,
      recordIP   : false,
      serverURLs : ''
    });
  }, window.Valine);
});
</script>

</body>
</html>
